# -*- coding: utf-8 -*-
import scrapy
from scrapy import Selector, Request

from dianying.items import DianyingItem
from scrapy.http import FormRequest
from scrapy.selector import SelectorList


class DianyingSpider(scrapy.Spider):
    name = 'dianying'
    allowed_domains = ['ygdy8.net']
    # allowed_domains = ['xiaohua.com']
    # start_urls = ['https://www.xiaohua.com/duanzi/']
    start_urls = ["https://www.ygdy8.net/html/gndy/dyzz/"]
    # start_urls = ["https://www.ygdy8.net/html/gndy/dyzz/list_23_2.html"]

    def __init__(self, start_url=None, db='console', *args, **kwargs):
        self.start_url = start_url
        self.db = db
        super().__init__(*args, **kwargs)

    def parse(self, response):
        from scrapy.http.response.html import HtmlResponse
        assert isinstance(response, HtmlResponse)
        item = response.xpath("//div[@class='co_content8']/ul")
        content = item.xpath(".//a")
        data = dict()
        # print(content.extract())
        for i in content:
            assert isinstance(i, scrapy.selector.unified.Selector)
            movie = i.xpath("./text()")[0].extract()
            from urllib.parse import urlparse
            p = urlparse(response.url)
            model = DianyingItem()
            model['movie'] = movie
            if 'href' in i.attrib:
                new_url = "{}://{}{}".format(p[0], p[1], i.attrib['href'])
                model['href'] = new_url
                data.update({"movie":movie,"href":new_url})
                yield model
        from urllib.parse import urlparse
        p = urlparse(response.url)
        page = response.xpath("//div[@class='x']")
        my_a = page.xpath(".//a")
        for i in my_a:
            if str(i.xpath("./text()").extract()[0])=='下一页':
                new_url = "{}://{}{}{}".format(p[0], p[1],'/html/gndy/dyzz/',i.attrib['href'])
                print(new_url)
                yield Request(new_url,callback = self.parse)

s = DianyingSpider()
